package com.ruoyi.nlp.buildwords;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import com.ruoyi.nlp.utils.MapOrderByValUtil;
import com.ruoyi.nlp.utils.MyThreadPool;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Splitter;

/**
 * 提取词
 * @author yjh
 *
 */
public class BuildNewWords {

	//存储每个文本的词频  及 左右单个字符
	//key:被切出来的词  value:map map包括 count:词频  leftS:左边单个字符  rightS:右边单个字符  ngd:凝固度 zyd:自由度
	private Map<String,Map<String,Object>> result = new HashMap<String,Map<String,Object>>();
	
	//提取的词
	private List<String> words = new ArrayList<String>();
	
	//分词总数
	private int total = 0;
	
	//线程池
	private MyThreadPool pool = new MyThreadPool(Config.threadSize);
	
	/**
	 * 执行方法
	 * @param inputString 待处理文本
	 * @throws IOException
	 */
	public void run (String inputString) throws IOException{
		handleText(inputString);
	}
	
	/**
	 * 执行方法
	 * @param inputFile  待处理文本文件
	 * @param charSet 文件编码
	 * @throws IOException
	 */
	public void run (File inputFile,String charSet) throws IOException{
		run(FileUtils.readFileToString(inputFile,charSet));
	}

	/**
	 * 执行方法
	 * @param inputFile  待处理文本文件(默认utf-8编码)
	 * @throws IOException
	 */
	public void run (File inputFile) throws IOException{
		run(FileUtils.readFileToString(inputFile,"utf-8"));
	}
	/**
	 * 处理文本 切词 计算凝固度 自由度
	 * @throws IOException 
	 */
	private void handleText(String text) throws IOException{
		//存储正序被切分的文本  如 中华人民
		List<String> left = new ArrayList<String>();
		
		//存储倒序被切分的文本  如 民人华中
		List<String> right = new ArrayList<String>();
		
		Map<String,String> reverse = new HashMap<String, String>();
		
		Map<String,Integer> reverse_count = new HashMap<String, Integer>();
		//过滤停用词及非汉字字符
		Iterable<String> texts = Splitter.onPattern("[^\u4E00-\u9FA5]|["+Config.stopwords+"]")
				.omitEmptyStrings().trimResults().split(text);
		for(String s:texts){
			for(int i=0;i<s.length();i++){
				total++;
				String leftToRight = s.substring(i,Math.min(Config.maxLen+i, s.length()));
				String rightToLeft = StringUtils.reverse(leftToRight);
				if(result.containsKey(leftToRight)){
					Map<String, Object> map = result.get(leftToRight);
					map.put("count", Integer.parseInt(map.get("count").toString())+1);
					reverse_count.put(rightToLeft, reverse_count.get(rightToLeft)+1);
				}else{
					if(leftToRight.length() > 1){
						left.add(leftToRight);
						right.add(rightToLeft);
						reverse.put(leftToRight, rightToLeft);
					}
					Map<String, Object> map = new HashMap<String, Object>();
					map.put("count", 1);
					result.put(leftToRight, map);
					reverse_count.put(rightToLeft, 1);
				}
			}
		}
		
		leftRight(left,right,reverse,reverse_count);
		ningGuDu();
		ziYouDu(reverse,reverse_count);
		filterAndSort();
	}
	
	/**
	 * 获取被切分文本的词频 及 左右单个字符
	 * @throws IOException 
	 */
	private void leftRight(final List<String> left,final List<String> right,final Map<String,String> reverse,final Map<String,Integer> reverse_count) throws IOException{
		//字典排序
		Collections.sort(left);
		Collections.sort(right);
		for(final String s:left){
			pool.execute(new Runnable() {
				public void run() {
					Map<String, Object> map = result.get(s);
					Map<String, Object> lMap = listLeftOrRightChar(left,s,null);
					map.put("rightS", lMap.get("chars"));
					map.put("rightC", lMap.get("count"));
					lMap = listLeftOrRightChar(right,reverse.get(s),reverse_count);
					map.put("leftS", lMap.get("chars"));
					map.put("leftC", lMap.get("count"));
					result.put(s, map);
				}
			});
		}
		pool.shutdown();
		while(true){
			if(pool.isTerminated()){
				break;
			}
			try {
				Thread.sleep(1);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
	}
	
	/**
	 * 获取左右单个字符
	 * @param list
	 * @param s
	 * @return
	 */
	private Map<String,Object> listLeftOrRightChar(List<String> list,String s,Map<String,Integer> reverse_count){
		Map<String,Object> map = new HashMap<String, Object>();
		List<String> charList = new ArrayList<String>();
		int size=0;
		for(int i=list.lastIndexOf(s)+1;i<list.size();i++){
			if(list.get(i).startsWith(s)){
				charList.add(list.get(i));
				if(reverse_count == null){
					size += Integer.parseInt(result.get(list.get(i)).get("count").toString());
				}else{
					size += reverse_count.get(list.get(i));
				}
			}else{
				break;
			}
		}
		map.put("chars", charList);
		map.put("count", size);
		return map;
	}
	/**
	 * 计算凝固度
	 */
	private void ningGuDu(){
		Set<String> keySet = result.keySet();
		for(String key:keySet){
			if(key.length() > 1){
				Map<String, Object> map = result.get(key);
				double keyP = Double.parseDouble(map.get("count").toString()) / total;
				long min = 0;
				if(key.length() != 1){
					//枚举左右部分
					for(int i=0;i<key.length()-1;i++){
						String before = key.substring(0, i+1);
						if(!result.containsKey(before)){
							continue;
						}
						String after = key.substring(i+1);
						if(!result.containsKey(after)){
							continue;
						}
						double beforeP = Double.parseDouble(result.get(before).get("count").toString()) / total;
						double afterP = Double.parseDouble(result.get(after).get("count").toString()) / total;
						
						//取最小凝固度值
						if(min == 0){
							min = Math.round(keyP / (beforeP * afterP));
						}else{
							min = Math.min(min, Math.round(keyP / (beforeP * afterP)));
						}
					}
					map.put("ngd", min);
				}
			}
		}
	}
	
	/**
	 * 计算自由度
	 */
	@SuppressWarnings("unchecked")
	private void ziYouDu(Map<String,String> reverse,Map<String,Integer> reverse_count){
		Set<Entry<String, Map<String, Object>>> entrySet = result.entrySet();
		for(Entry<String, Map<String, Object>> entry:entrySet){
			if(entry.getKey().length() > 1){
				Map<String, Object> value = entry.getValue();
				double leftLog = leftZyd((List<String>)value.get("leftS"),reverse_count,entry.getKey());
				double rightLog = rightZyd((List<String>)value.get("rightS"),entry.getKey());
				value.put("zyd", Math.min(leftLog, rightLog));
			}
		}
	}
	
	/**
	 * 计算左信息熵
	 * @param lOrRList
	 * @return
	 */
	private double leftZyd(List<String> lOrRList,Map<String,Integer> reverse_count,String key){
		int size = Integer.parseInt(result.get(key).get("leftC").toString());
		double lOrRlog = 0.0;
		for(String s:lOrRList){
			double value = (double)reverse_count.get(s);
			lOrRlog -= (value / size) * Math.log(value / size);
		}
		return lOrRlog;
	}
	
	/**
	 * 计算左信息熵
	 * @param lOrRList
	 * @return
	 */
	private double rightZyd(List<String> lOrRList,String key){
		int size = Integer.parseInt(result.get(key).get("rightC").toString());
		double lOrRlog = 0.0;
		for(String s:lOrRList){
			double value = Double.parseDouble(result.get(s).get("count").toString());
			lOrRlog -= (value / size) * Math.log(value / size);
		}
		return lOrRlog;
	}
	/**
	 * 过滤及排序
	 */
	private void filterAndSort(){
		Set<Entry<String, Map<String, Object>>> entrySet = result.entrySet();
		Map<String,Integer> map = new HashMap<String, Integer>();
		for(Entry<String, Map<String, Object>> entry:entrySet){
			Map<String, Object> value = entry.getValue();
			if(entry.getKey().length() >1 && Integer.parseInt(value.get("count").toString()) >= Config.minCiPin
					&& Integer.parseInt(value.get("ngd").toString()) >= Config.minWhole 
					&& Double.parseDouble(value.get("zyd").toString()) >= Config.minLeftRight){
				map.put(entry.getKey(), Integer.parseInt(value.get("count").toString()));
			}
		}
		//排序
		words = MapOrderByValUtil.getTopKey(map, 0, true);
		
	}
	
	/**
	 * 原始处理结果保存成文件
	 * @param isOnlyHaveWord 是否只输出词(不包含词频 凝固度 自由度这些值)
	 * @throws IOException 
	 */
	public void saveFile(String filePath,boolean isOnlyHaveWord) throws IOException{
		
		File f = new File(filePath);
		if(f.isDirectory()){
			System.out.println("不能传入目录");
			return;
		}
		List<String> list = words;
		if(!isOnlyHaveWord){
			list = new ArrayList<String>();
			for(String word:words){
					word += "~"+result.get(word).get("count")+"~"+result.get(word).get("ngd")
							+"~"+result.get(word).get("zyd");
				list.add(word);
			}
		}
		
		FileUtils.writeLines(f, "utf-8", list);
	}
	
	/**
	 * 原始处理结果保存成文件（只包含词）
	 * @throws IOException 
	 */
	public void saveFile(String filePath) throws IOException{
		
		saveFile(filePath,true);
	}
	
	/**
	 * 对结果进一步过滤
	 * @param ingputPath
	 * @param outPath
	 * @throws IOException
	 */
	public void filterResult(String inputPath,String outPath) throws IOException{
		
		File input = new File(inputPath);
		File output = new File(outPath);
		if(input.isDirectory() || output.isDirectory()){
			System.out.println("参数不能传入目录");
			return;
		}
		List<String> lines = FileUtils.readLines(input, "utf-8");
		List<String> newLines = new ArrayList<String>();
		for(String line:lines){
			//词_词频_凝固度_自由度
			String[] w_c_n_z = line.split("~");
			//取出满足条件的词
			if(Integer.parseInt(w_c_n_z[1]) >= Config.minCiPin 
					&& Integer.parseInt(w_c_n_z[2]) >= Config.minWhole && 
					Double.parseDouble(w_c_n_z[3]) >= Config.minLeftRight){
				newLines.add(line);
			}
		}
		FileUtils.writeLines(output, "utf-8", newLines);
	}
	
	/**
	 * 获取提取的词集合
	 * @return
	 */
	public List<String> getWords(){
		return words;
	}
	
	/**
	 * 获取提取词map集合  包含词 词频 自由度 凝固度
	 * @return
	 */
	public List<Map<String,Object>> getWordsMap(){
		List<Map<String,Object>> list = new ArrayList<Map<String,Object>>();
		for(String word:words){
			Map<String,Object> map = new HashMap<String, Object>();
			map.put("word", word);
			map.put("cipin", result.get(word).get("count"));
			map.put("ngd", result.get(word).get("ngd"));
			map.put("zyd", result.get(word).get("zyd"));
			list.add(map);
		}
		return list;
	}
}
